package com.ossean.util;

import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import javax.annotation.Resource;

import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;

import com.ossean.databaseDest.DBDest;
import com.ossean.databaseSource.DBSource;
import com.ossean.databaseSource.GatherDao;
import com.ossean.model.EddRelations;
import com.ossean.model.GatherProjectsModel;
import com.ossean.model.Synonymmings;
import com.ossean.model.Synonyms;
import com.ossean.model.Taggings;
import com.ossean.model.Tags;

@Component("mergeProjectsUtil")
public class MergeProjectsUtil {
	Logger logger = Logger.getLogger(this.getClass());
	@Resource
	private DBSource dbSource;
	@Resource
	private GatherDao gatherDao;
	@Resource
	private DBDest dbDest;
	@Qualifier("gatherProjectsUtil")
	@Autowired
	private GatherProjectsUtil gatherProjectsUtil;
	
	private String synonymTableName = "synonyms";
	private String synonymmingTableName = "synonymmings";
	private String gatherProjectsTableName = "gather_projects";
	private String pointerTableName = "edd_pointers";
	
	private String taggingsTableName = "taggings";
	private String tagsTableName = "tags";
	
	private String eddRelationTableName = "edd_relations";
	
	
	//判断两个项目是否相同 默认两个项目的urlMD5码是不同的
	public  boolean areTwoProjectsTheSame(GatherProjectsModel model1, GatherProjectsModel model2){
		/**
		 * 如果是同一个社区 则默认不相同
		 */
		if(model1.getSource().equals(model2.getSource()))
			return false;
		boolean result = false;
		//两个项目初步匹配结果是相同的
		boolean stepOneResult = false;
		result = isTheSameAfterTFIDF(model1, model2, stepOneResult);
		return result;
	}
	
	
	//edd算法提取linkNames 其中包含项目名本身
	public List<String> getSynonyms(GatherProjectsModel model){
		
		model.setName(model.getName().trim());
		model.setDescription(model.getDescription().trim());
		
		List<String> result = dbSource.getSynonyms(synonymTableName, synonymmingTableName, model.getId());//表示当前的项目已经处理过了 当前的是需要更新的
		if(result.size() != 0){
			for(String s : result){
				logger.info("gather_project:" + model.getId() + " gets synonyms:" + s + " in table");
			}
			return result;	//表示当前项目的别名在数据库中已经存在了
		}
		String description = model.getDescription();
		if(description == null)
			return result;

		Map<String,Integer> linkNames = new HashMap<String,Integer>();//用于存储所有提取出来的关联项目名
		//对项目名进行处理
		//提取项目名中去除公司名或基金组织的信息
		String projectNameWithoutComName = StringHandler_ProjectName.getProjectWithoutComName(model.getName());
		if(!"".equals(projectNameWithoutComName)){
			//表示匹配到了相应公司或基金委的项目
			linkNames.put(projectNameWithoutComName, 1);
		}
		
		//提取项目名括号中的信息
		String bracket = StringHandler_ProjectName.getBracket(model.getName());
		if(!"".equals(bracket)){
			//去除括号中信息的特殊符号
			bracket = RegexHandler.extractEngDecChiAndDot(bracket);
			if(!RegexHandler.onlySpecialChar(bracket)){
System.out.println("项目括号中的信息： "+bracket);
				linkNames.put(bracket, 1);
			}
		}
		
		//读取be动词表
		List<String> beWords = FileReader.read("./files/beVerb.txt");
		description = StringHandler.getFirstSentence(description);
		for(String beWord:beWords){
			//按系动词表优先级进行匹配
			String linkName = StringHandler.findLinkName(description, beWord);
System.out.println("desc be动词之前的： "+linkName);
			if(linkName == null){
				//表示没有匹配到当前的系动词 进行下一个匹配
				continue;
			}
			//特征短语去噪
			//提取主副描述信息(括号中)
			List<String> linkNameResult = EDDHandler.getMainAndViceDescriptionByBracket(linkName);
			//提取主副描述信息(known as+连词)
			linkNameResult = EDDHandler.getMainAndViceDescriptionByKnowAs(linkNameResult);
			//去除定语描述信息(逗号定语)
			linkNameResult = EDDHandler.removeComma(linkNameResult);
			//去除定语描述信息(定冠词The)
			linkNameResult = EDDHandler.removeDemonstrativeWords(linkNameResult);
			//去除从句描述信息
			linkNameResult = EDDHandler.removeArrtibutiveClause(linkNameResult);
			//去除指示代词
			linkNameResult = EDDHandler.removePronoun(linkNameResult);
			//去除项目常用词
			linkNameResult = EDDHandler.removeProject(linkNameResult);
			//去除指示代词和项目常用词两者的笛卡尔积
			linkNameResult = EDDHandler.removePPCombine(linkNameResult);
//			//特殊字符处理(空字符串)
//			linkNameResult = EDDHandler.removeEmpty(linkNameResult);
//			//特殊字符处理(引号)
//			linkNameResult = EDDHandler.removeSemicolon(linkNameResult);
//			特殊字符处理(除英文中文外 中间用空格连接)
//			linkNameResult = EDDHandler.removeSingleColon(linkNameResult);
			//提取项目中
			linkNameResult = RegexHandler.extractEngDecChiAndDot(linkNameResult);
			//删除只有特殊字符的同义词
			linkNameResult = RegexHandler.removeOnlySpecial(linkNameResult);
			for(String name:linkNameResult){
				if(linkNames.containsKey(name)){
					linkNames.put(name, linkNames.get(name)+1);
				}else{
					if(!name.equals(model.getName().toLowerCase()))
						linkNames.put(name, 1);
				}
			}
			break;
		}//遍历be动词 for循环结尾
		
		//将map转换成List
		Set<String> keys = linkNames.keySet();
		Iterator<String> it = keys.iterator();
		while(it.hasNext()){
			String a = it.next();
			result.add(a);
System.out.println("result中包含的： "+a);
		}
		
		/**
		 * 对上面提取出来的同义词结果进行进一步限制 返回结果为result3
		 */
		List<String> result2 = new ArrayList<String>();
		//若项目名为缩写形式 项目描述信息中存不存在项目名的全称
		
		String fullName = StringHandler.getFullName(model.getName().toLowerCase(), result);
		if(!fullName.equals("")){
			result2.add(fullName.toLowerCase());//表示项目名确实是缩写 并提取到了项目全名
System.out.println("项目的全称为： "+fullName);
		}
		
		
		String shortName = StringHandler.getShortName(model.getName().toLowerCase(), result);
		if(!shortName.equals("")){
			result2.add(shortName.toLowerCase());//表示项目名确实是全称 并提取到了项目缩写
System.out.println("项目的简称为： "+shortName);
		}
		
		//要求描述信息中出现的项目名对应的词 按照顺序链接到一起
		for(int i = 0; i < result.size(); i++){
			//同义词要在项目名中出现
			String extract = "";
			String synonym = result.get(i);
			String[] words = synonym.split(" ");//按照空格进行分词
			//去掉括号检测同义词是否在项目名中出现
			String modelNameWithoutBracket = StringHandler_ProjectName.removeBracket(model.getName().toLowerCase());
System.out.println("modelNameWithoutBracket : "+modelNameWithoutBracket);
			for(String word:words){
				//查看每个词在项目名中是否出现
				if(modelNameWithoutBracket.indexOf(word) >= 0){
					//表示项目名中存在这个分词
					extract += word + " ";
				}
			}
			if(!"".equals(extract)){
				//表示确实是别名
				extract = extract.substring(0, extract.length() - 1);
System.out.println("别名在项目名中有出现： "+extract);
				result2.add(extract);
				continue;
			}
		}
		
		//对最终结果进行处理：如果与项目原名相同：去除 如果提取出了相同的结果：去除重复的
		Map<String,Integer> tmp = new HashMap<String,Integer>();
		for(String r:result2){
			r = r.toLowerCase();
			if(!r.equals(model.getName().toLowerCase()))
				tmp.put(r, 1);
		}
		//将map中的key转换为list
		List<String> result3 = new ArrayList<String>();
		Set<String> keys_tmp = tmp.keySet();
		Iterator it_tmp = keys_tmp.iterator();
		while(it_tmp.hasNext())
			result3.add(it_tmp.next().toString());
		result3.add(model.getName().toLowerCase());
		//项目同义词不包括项目原名
		
		
		//输出日志文件
		for(String s : result3){
			logger.info("gather_project:" + model.getId() + " gets synonyms:" + s + " by description");
		}
		return result3;
	}
	

	//根据项目别名判断两个项目初步匹配是否正确 返回值为匹配上的项目别名
	public boolean isTheSameByLinkName(List<String> linkNames1, List<String> linkNames2){
		//对两个项目的别名之间进行匹配
		for(int i = 0; i < linkNames1.size(); i++){
			String linkName1 = linkNames1.get(i);
			for(int j = 0; j < linkNames2.size(); j++){
				String linkName2 = linkNames2.get(j);
				if(linkName1.equals(linkName2))
					return true;
			}
		}
		return false;
	}


	//去伪取真过程
	public boolean isTheSameAfterTFIDF(GatherProjectsModel model1, GatherProjectsModel model2, boolean stepOneResult){
		//获取两个对象的tags属性
		String tags1 = model1.getTags();
		String tags2 = model2.getTags();
		if((tags1 != null && tags2 != null)&&(!"".equals(tags1) && !"".equals(tags2))){
			//表示两个项目都有标签
			double similarity = calSimilarityByTag(model1, model2);
System.out.println("calSimilarityByTag"+String.valueOf(similarity));
			if(stepOneResult){
				//表示两个项目源自一个社区 去伪
				//如果两个项目匹配结果 weight>0.9 表示是同一个项目
				if(similarity > 0.9)
					return true;
				return false;
			}else{
				//表示两个项目不来自同一个社区 存真
				//如果两个项目匹配结果 weight<0.1 表示不是同一个项目
				if(similarity < 0.1)
					return false;
				return true;
			}
		}
		else{
			if((tags1 == null && tags2 != null) || ("".equals(tags1) && !"".equals(tags2)) || (!"".equals(tags1) && "".equals(tags2)) || (tags1 != null && tags2 == null)){
				//表示两者有一个项目有标签
				Map<String, Integer> result = new HashMap<String, Integer>();
				List<String> tagList = new ArrayList<String>();
				int count_tag = 0;
				if(tags1 == null){
					//表示项目2有标签
					//对项目1的描述信息进行分词
					result = getResolveResult(model1.getDescription());
					String[] tmp = tags2.split(",");
					count_tag = tmp.length;
					tagList = changeArrayToList(tmp);
				}else{
					//表示项目1有标签
					result = getResolveResult(model2.getDescription());
					String[] tmp = tags1.split(",");
					count_tag = tmp.length;
					tagList = changeArrayToList(tmp);
				}
				if(stepOneResult){
					//表示两个项目来自同一社区 匹配上的标签数量应该在50%以上
					int threshold = 0;
					if(count_tag % 2 == 0)
						threshold = count_tag / 2;
					else
						threshold = count_tag / 2 + 1;
					int count_match = matchListAndMap(tagList,result);
					if(count_match >= threshold)
						return true;
					else
						return false;
				}else{
					int threshold = 1;
					int count_match = matchListAndMap(tagList, result);
					if(count_match >= threshold)
						return true;
					else
						return false;
				}
			}else{
				//表示两者都没有标签
				double similarity = calSimilarityByDescription(model1, model2);
				if(stepOneResult){
					//表示两个项目源自一个社区 去伪
					//如果两个项目匹配结果 weight>0.9 表示是同一个项目
					if(similarity > 0.9)
						return true;
					return false;
				}else{
					//表示两个项目不来自同一个社区 存真
					//如果两个项目匹配结果 weight<0.1 表示不是同一个项目
					if(similarity < 0.1)
						return false;
					return true;
				}
			}
		}
	}
	
	
	//计算都有标签的项目之间的相似度
	public double calSimilarityByTag(GatherProjectsModel model1, GatherProjectsModel model2){
//		读取item_tag_relation表中项目总数
//		int wholeNum = dbSource.getItemNumber(taggingsTableName, "OpenSourceProject");
//		读取item_tag_relation表中该model对应的标签对象
		
//		List<Taggings> taggingsList1 = dbSource.getTaggingsListByTaggableId(taggingsTableName, model1.getId());
//		List<Taggings> taggingsList2 = dbSource.getTaggingsListByTaggableId(taggingsTableName, model2.getId());
//		
//		//读取每个name对应的tag对象
//		List<Tags> tagList1 = getTagListByItemTagRelationList(taggingsList1);
//		List<Tags> tagList2 = getTagListByItemTagRelationList(taggingsList2);
		
		//获取权重向量
//		Map<Integer, List<Double>> weightListMap = getWeightVectorByTag(tagList1, tagList2, wholeNum);
//		List<Double> weightVector1 = weightListMap.get(1);
//		List<Double> weightVector2 = weightListMap.get(2);
//		return calDistance(weightVector1, weightVector2);
		
		String[] tagArray_model1 = model1.getTags().split(",");
		String[] tagArray_model2 = model2.getTags().split(",");
		List<String> tags_model1 = changeArrayToList(tagArray_model1);
		List<String> tags_model2 = changeArrayToList(tagArray_model2);
		
		//比较两个list中相同标签的个数
		List<String> sameTags = getSameStringsInList(tags_model1, tags_model2);
		int minNum_tag = tags_model1.size() < tags_model2.size() ? tags_model1.size() : tags_model2.size();
		float sameRate = (float)(sameTags.size() * 1.0 / minNum_tag);
System.out.println("same rate between teo tags : "+sameRate);
		return sameRate;
	}
	
	//计算文本相似度
	public double calSimilarityByDescription(GatherProjectsModel model1, GatherProjectsModel model2){
		long time4_1 = System.currentTimeMillis();
		String description1 = model1.getDescription();
		String description2 = model2.getDescription();
		Map<String, Integer> resolveResult1 = getResolveResult(description1);
		Map<String, Integer> resolveResult2 = getResolveResult(description2);
		Map<String, Double> tfResult1 = calTFOfDescription(resolveResult1);
		Map<String, Double> tfResult2 = calTFOfDescription(resolveResult2);
		Map<String, Double> idfResult = calIDFOfDescription(resolveResult1, resolveResult2);
		
		List<String> commonList = new ArrayList<String>();
		List<String> uniqueList1 = new ArrayList<String>();
		List<String> uniqueList2 = new ArrayList<String>();
		
		Set<String> keys1 = resolveResult1.keySet();
		Iterator<String> it1 = keys1.iterator();
		while(it1.hasNext()){
			String key1 = it1.next();
			if(resolveResult2.containsKey(key1)){
				commonList.add(key1);
			}else{
				uniqueList1.add(key1);
			}
		}
		
		Set<String> keys2 = resolveResult2.keySet();
		Iterator<String> it2 = keys2.iterator();
		while(it2.hasNext()){
			String key1 = it2.next();
			if(resolveResult1.containsKey(key1)){
				//因为遍历第一个map时已经将commonList进行构造了 此处不用再add了
			}else{
				uniqueList2.add(key1);
			}
		}
		
		List<Double> weightCommon1 = calWeightListOfDescription(commonList, tfResult1, idfResult);
		List<Double> weightCommon2 = calWeightListOfDescription(commonList, tfResult2, idfResult);
		List<Double> weightUnique1 = calWeightListOfDescription(uniqueList1, tfResult1, idfResult);
		List<Double> weightUnique2 = calWeightListOfDescription(uniqueList2, tfResult2, idfResult);
		
		List<Double> weightResult1 = new ArrayList<Double>();
		List<Double> weightResult2 = new ArrayList<Double>();
		
		for(int i = 0; i < weightCommon1.size(); i++){
			weightResult1.add(weightCommon1.get(i));
			weightResult2.add(weightCommon2.get(i));
		}
		
		for(int i = 0; i < weightUnique1.size(); i++){
			weightResult1.add(weightUnique1.get(i));
			weightResult2.add(0.0);
		}
		
		for(int i = 0; i < weightUnique2.size(); i++){
			weightResult1.add(0.0);
			weightResult2.add(weightUnique2.get(i));
		}
		long time4_2 = System.currentTimeMillis();
		if(time4_2 - time4_1 > 10)
			logger.info("time cost for cal similarity by description: " + (time4_2 - time4_1));
		return calDistance(weightResult1, weightResult2);
	}
	
	
	
	//找两个向量组成的相似向量
	public Map<Integer, List<Double>> getWeightVectorByTag(List<Tags> tagList1, List<Tags> tagList2, int wholeNum){
		long time5_1 = System.currentTimeMillis();
		List<Tags> commonTagList1 = new ArrayList<Tags>();//里面用Tag表示格式不同的相同标签 在取得时候两者都要取出来
		List<Tags> commonTagList2 = new ArrayList<Tags>();
		List<Tags> uniqueTag1 = new ArrayList<Tags>();
		List<Tags> uniqueTag2 = new ArrayList<Tags>();
		//首先要根据两边的tag组成向量
		for(int i = 0; i < tagList1.size(); i++){
			Tags tag1 = tagList1.get(i);
			Tags similarTag;
			if((similarTag = theMostSimilarTag(tag1, tagList2)) != null){
				//表示存在相同的标签
				commonTagList1.add(tag1);
				commonTagList2.add(similarTag);
			}
		}
		
		//找到各自独有的标签
		uniqueTag1 = findUniqueTag(tagList1, commonTagList1);
		uniqueTag2 = findUniqueTag(tagList2, commonTagList2);
		
		List<Double> weightList1 = new ArrayList<Double>();
		List<Double> weightList2 = new ArrayList<Double>();
		
		for(int i = 0; i < commonTagList1.size(); i++){
			Tags tag1 = commonTagList1.get(i);
			Tags tag2 = commonTagList2.get(i);
			double weight = calIDFOfTag(tag1, tag2, wholeNum);
			weightList1.add(weight);
			weightList2.add(weight);
		}
		
		for(int i = 0; i < uniqueTag1.size(); i++){
			Tags tag1 = uniqueTag1.get(i);
			double weight = calIDFOfTag(tag1, wholeNum);
			weightList1.add(weight);
			weightList2.add(0.0);
		}
		
		for(int i = 0; i < uniqueTag2.size(); i++){
			Tags tag2 = uniqueTag2.get(i);
			double weight = calIDFOfTag(tag2, wholeNum);
			weightList2.add(weight);
			weightList1.add(0.0);
		}
		
		Map<Integer, List<Double>> result = new HashMap<Integer, List<Double>>();
		result.put(1, weightList1);
		result.put(2, weightList2);
		long time5_2 = System.currentTimeMillis();
		if(time5_2 - time5_1 > 10)
			logger.info("time cost for get weight vector by tag: " + (time5_2 - time5_1));
		return result;
	}
	
	
	//找List各自独有的标签
	public List<Tags> findUniqueTag(List<Tags> tagList, List<Tags> commonList){
		List<Tags> result = new ArrayList<Tags>();
		for(Tags tag:tagList){
			if(!isExist(tag, commonList)){
				//表示独有
				result.add(tag);
			}
		}
		return result;
	}
	
	//判断后者是否拥有前者
	public boolean isExist(Tags tag, List<Tags> list){
		for(Tags t:list){
			if(tag.getId() == t.getId())
				return true;
		}
		return false;
	}
	
	
	
	//找与该标签最相似的标签
	public Tags theMostSimilarTag(Tags tag, List<Tags> tagList){
		if(tag == null)
			return null;
		//对标签进行特殊字符去除处理
		String tagName = RegexHandler.extractEngDecimalAndChinese(tag.getName());
		for(Tags t:tagList){
			if(t == null)
				continue;
			String tName = RegexHandler.extractEngDecimalAndChinese(t.getName());
			if(tagName.equals(tName)){
				//表示两个标签完全相同
				return t;
			}
		}
		return null;//表示没有找到相同的标签
	}
	
	
	public String theMostSimilarString(String str, List<String> strList){
		if(str == null)
			return null;
		for(String s : strList){
			if(s == null)
				continue;
			if(s.equals(str)){
				//表示两个str相同
				return str;
			}
		}
		return null;
	}


	//计算tag对应id的idf值
	public double calIDFOfTag(Tags tag1, Tags tag2, int wholeNum){
		double result = 0.00;
		int count = dbSource.getNumOfTagIdAndOspByTwoTagIds(taggingsTableName, tag1.getId(), tag2.getId());
		result = wholeNum * 1.0 / count;
		result = Math.log(result);
		return result;
	}
	//上面函数的重载
	public double calIDFOfTag(Tags tag, int wholeNum){
		double result = 0.00;
		int count = dbSource.getNumOfTagIdAndOsp(taggingsTableName, tag.getId());
		result = wholeNum * 1.0 / count + 0.0001;
		result = Math.log(result);
		return result;
	}
	
	//根据itemTagRelationList找到对应tag_id的Tag对象列表
	public List<Tags> getTagListByItemTagRelationList(List<Taggings> itemTagRelationList){
		List<Tags> result = new ArrayList<Tags>();
		for(Taggings i:itemTagRelationList){
			Tags t = dbSource.getTagById(tagsTableName, i.getTag_id());
			result.add(t);
		}
		return result;
	}


	//计算向量余弦夹角
	public double calDistance(List<Double> vector1, List<Double> vector2){
		double numerator = 0.0;//分子
		double denominator = 0.0001;//分母
		numerator = calNumerator(vector1, vector2);
		denominator = calDenominator(vector1, vector2);
		return numerator / denominator;
	}
	//余弦夹角分子计算
	public double calNumerator(List<Double> vector1, List<Double> vector2){
		Double result = 0.0;
		for(int i = 0; i < vector1.size(); i++){
			result += vector1.get(i) * vector2.get(i);
		}
		return result;
	}
	//余弦夹角分母计算
	public double calDenominator(List<Double> vector1, List<Double> vector2){
		Double result = 0.0001;
		double result1 = 0.0;
		for(Double d:vector1){
			result1 += d * d;
		}
		result1 = Math.sqrt(result1);
		
		double result2 = 0.0;
		for(Double d:vector2){
			result2 += d * d;
		}
		result2 = Math.sqrt(result2);
		result = result1 * result2;
		return result;
	}


	//对描述信息进行分词得到词频统计结果的Map
	public Map<String, Integer> getResolveResult(String description){
		long time6_1 = System.currentTimeMillis();
		Map<String, Integer> result = new HashMap<String, Integer>();
		Reader text = new StringReader(description);
		IKSegmenter ik = new IKSegmenter(text, true);
		Lexeme term = null;
		try {
			//处理一个分词结果
			while((term = ik.next()) != null){
				String tmp = term.getLexemeText().toString();
				tmp = RegexHandler.extractEngDecimalAndChinese(tmp).toLowerCase();
				if(result.containsKey(tmp))
					result.put(tmp, result.get(tmp) + 1);
				else
					result.put(tmp, 1);
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		long time6_2 = System.currentTimeMillis();
		if(time6_2 - time6_1 > 10)
			logger.info("time cost for get resolve result: " + (time6_2 - time6_1));
		return result;
	}
	
	
	//计算文本匹配tf值
	public Map<String, Double>  calTFOfDescription(Map<String, Integer> map){
		long time7_1 = System.currentTimeMillis();
		Map<String, Double> result = new HashMap<String, Double>();
		Set<String> keys = map.keySet();
		Iterator<String> it = keys.iterator();
		int tfAll = 0;
		//计算当前描述所有词词频总和
		while(it.hasNext()){
			String name = it.next();
			tfAll += map.get(name);
		}
		Iterator<String> it2 = keys.iterator();
		while(it2.hasNext()){
			String name = it2.next();
			int tf = map.get(name);
			double weight = tf * 1.0 / tfAll;
			result.put(name, weight);
		}
		long time7_2 = System.currentTimeMillis();
		if(time7_2 - time7_1 > 10)
			logger.info("time cost for cal tf of description: " + (time7_2 - time7_1));
		return result;
	}
	
	
	//计算文本匹配IDF值
	public Map<String, Double> calIDFOfDescription(Map<String, Integer> map1, Map<String, Integer> map2){
		long time8_1 = System.currentTimeMillis();
		int docNum = 2;
		
		Map<String, Double> result = new HashMap<String, Double>();
		Map<String, Integer> count_n = new HashMap<String, Integer>();
		Set<String> keys1 = map1.keySet();
		Iterator<String> it1 = keys1.iterator();
		while(it1.hasNext()){
			String key = it1.next();
			if(count_n.containsKey(key)){
				count_n.put(key, count_n.get(key) + 1);
			}else{
				count_n.put(key, 1);
			}
		}
		
		Set<String> keys2 = map2.keySet();
		Iterator<String> it2 = keys2.iterator();
		while(it2.hasNext()){
			String key = it2.next();
			if(count_n.containsKey(key)){
				count_n.put(key, count_n.get(key) + 1);
			}else{
				count_n.put(key, 1);
			}
		}
		
		Set<String> keys = count_n.keySet();
		Iterator<String> it = keys.iterator();
		while(it.hasNext()){
			String name = it.next();
			double weight = docNum / count_n.get(name) * 1.0 + 0.0001;
			weight = Math.log(weight);
			result.put(name, weight);
		}
		long time8_2 = System.currentTimeMillis();
		if(time8_2 - time8_1 > 10)
			logger.info("time cost for cal idf of description: " + (time8_2 - time8_1));
		return result;
	}
	
	
	//根据文本的list得到weight的list
	public List<Double> calWeightListOfDescription(List<String> textList, Map<String, Double> tfMap, Map<String, Double> idfMap){
		long time9_1 = System.currentTimeMillis();
		List<Double> result = new ArrayList<Double>();
		for(int i = 0; i < textList.size(); i++){
			double weight = tfMap.get(textList.get(i)) * idfMap.get(textList.get(i));
			result.add(weight);
		}
		long time9_2 = System.currentTimeMillis();
		if(time9_2 - time9_1 > 10)
			logger.info("time cost for cal weight list of description: " + (time9_2 - time9_1));
		return result;
	}
	
	
	
	//将linkNames存入数据库
	public void insertSynonyms(List<String> linkNames){
		for(String name:linkNames){
			Synonyms model = new Synonyms();
			model.setName(name.trim());
			dbSource.insertSynonym(synonymTableName, model);
		}
	}
	
	
	//插入数据到同义词表和项目关联表
	public void insertSynonymming(List<String> linkNames, int synonymming_id){
		for(String name:linkNames){
			//查找同义词对象
			Synonyms synonyms = dbSource.getSynonymByName(synonymTableName, name);
			//如果synonyms漏掉 没有加入到数据库 则不执行后面的操作
			if(synonyms == null)
				continue;
			Synonymmings synonymmings = new Synonymmings();
			synonymmings.setSynonym_id(synonyms.getId());
			synonymmings.setSynonymming_id(synonymming_id);
			try {
				dbSource.insertSynonymming(synonymmingTableName, synonymmings);
				logger.info("project:" + synonymming_id + " has synonym:" + synonyms.getName());
			} catch (Exception e) {
				//在插入同义词时 gather_projects中的记录已经被删除掉了
				logger.info("gather_projects_id has been removed");
				break;
			}
		}
	}

	
	//匹配完成后 将关联上的项目进行合并 结果存储到edd_relations表中 最后一个参数为true表示是更新表中的数据
	public void handleAfterMatch(GatherProjectsModel model, List<String> synonyms, List<GatherProjectsModel> matchedProjectList, boolean updateOrNot){
		insertSynonyms(synonyms);//插入新增项目的同义词
		insertSynonymming(synonyms, model.getId());//插入新增项目和同义词之间的关联
		
		//处理同义词表
		for(int i = 0; i < matchedProjectList.size(); i++){
			GatherProjectsModel model1 = matchedProjectList.get(i);
			List<String> synonyms_1 = getSynonyms(model1);//获取同义词
			//判断两个项目是不是来自同一个社区 如果是则不插入彼此的同义词
			if(!((""+model.getSource()).equals(model1.getSource()+""))){
				logger.info("project:" + model.getId() + " and project:" + model1.getId() + " add synonyms after match");
				insertSynonymming(synonyms_1, model.getId());//在新增项目中插入关联项目同义词记录
				insertSynonymming(synonyms, model1.getId());//在关联项目中插入新增项目的同义词记录
			}
		}
		
		List<EddRelations> relationsList = new ArrayList<EddRelations>();
		EddRelations relation_model = new EddRelations();
		relation_model.setGather_projects_ids(","+model.getId()+",");
		relationsList.add(relation_model); //relationsList用于记录包含当前项目在内的
		
		List<EddRelations> relationsList_matched = new ArrayList<EddRelations>();//匹配上的项目在edd_relations表中的记录
		for(int i = 0; i < matchedProjectList.size(); i++){
			GatherProjectsModel model2 = matchedProjectList.get(i);
			//查找edd_relations表得到相应的记录
			List<EddRelations> eddRelationList = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, model2.getId());
			for(EddRelations relation:eddRelationList){
				relationsList.add(relation);
				relationsList_matched.add(relation);
			}
			if(eddRelationList.size() == 0){
				//表示之前没有匹配上项目的匹配记录 需要插入这条记录 eg: ,12,
				EddRelations relation = new EddRelations();
				relation.setGather_projects_ids(","+model2.getId()+",");
				relationsList.add(relation); //relationsList用于记录包含当前项目在内的
			}
		}
		
		//relationsList_now表示目前正在处理的项目在edd_relations表中的记录 需要在插入新的edd_relations对象前查找
		List<EddRelations> relationsList_now = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, model.getId());
		
		//统计relationList中所有不同的gather_projects_id
		String relation_new_gather_projects_ids = hasDiffProjFromSameCommunity(relationsList);
		EddRelations eddRelation = new EddRelations();
		if(relation_new_gather_projects_ids != null){
			//表示匹配结果中不存在同一个社区不同的多个项目匹配到一起
			eddRelation.setGather_projects_ids(relation_new_gather_projects_ids);
			dbSource.insertEddRelations(eddRelationTableName, eddRelation);//插入最新的结果集
			/**
			 * 删除关联项目在edd_relations表中对应的记录
			 */
			for(EddRelations relation:relationsList_matched)
				deleteExistEddRelations(relation);
			/**
			 * 如果是更新项目 需要查看edd_relations表中是否有该记录 如果有要删除掉
			 */
			for(EddRelations relation:relationsList_now)
				deleteExistEddRelations(relation);
		}else{
			//表示结果有来自同一社区的项目 如果当前项目在edd_relations表中找不到相应记录 则应插入本身的记录；反之不进行操作
			if(relationsList_now.size() == 0){
				//表示不是更新的项目 需要插入当前项目本身的记录
				dbSource.insertEddRelations(eddRelationTableName, relation_model);
			}
		}
		
		
	}
	
	
	public int changeNullToInt(Integer i){
		if(i == null)
			return 0;
		else 
			return i;
	}
	
	public List<String> changeArrayToList(String[] strs){
		List<String> result = new ArrayList<String>();
		for(String str:strs){
			str = RegexHandler.extractEngDecimalAndChinese(str);//对标签进行处理
			result.add(str);
		}
		return result;
	}
	
	//匹配List中的数据和Map中的数据 返回匹配上得数量
	public int matchListAndMap(List<String> tags, Map<String, Integer> description){
		int count = 0;
		for(String tag:tags){
			if(description.containsKey(tag))
				count++;
		}
		return count;
	}
	
	//找到当前项目同义词关联的项目 要求项目不相同 因为同一个项目可能包含多个同义词
	public List<GatherProjectsModel> getRelatedGatherProjects(GatherProjectsModel model, List<String> synonymList){
		List<GatherProjectsModel> result = new ArrayList<GatherProjectsModel>();
		for(String name:synonymList){
			Synonyms synonym = dbSource.getSynonymByName(synonymTableName, name);
			if(synonym == null)
				continue;//表示还不存在这样的标签
			List<Synonymmings> synonymmingList = dbSource.selectSynonymmingListBySynonymId(synonymmingTableName, synonym.getId());
			for(Synonymmings synonymming:synonymmingList){
				GatherProjectsModel m = gatherDao.selectGPMById(gatherProjectsTableName, synonymming.getSynonymming_id());
				if(model.getId() != m.getId()){
					//判断m在result中是否存在
					int i;
					for(i = 0; i < result.size(); i++){
						GatherProjectsModel tmp = result.get(i);
						if(tmp.getId() == m.getId())
							break;
					}
					if(i == result.size()){
						result.add(m);
						System.out.println("project:" + model.getId() + " match project:" + m.getId() + " by synonym:" + name);
					}
				}
			}
		}
		return result;
	}
	
	//对url地址中的特殊字符进行转义处理
	public String changeUrl(String url){
		url = url.replace("/", "//");
		url = url.replace("%", "/%");
		url = url.replace("_", "/_");
		url = url.replace("\\", "/\\");
		return url;
	}
	
	
	//判断项目名的缩写是不是别名
	public String getShortProjectName(String name, List<String> list){
		String[] tmp = name.toLowerCase().split(" ");
		String shortName = "";
		for(String s:tmp){
			String firstLetter = s.substring(0, 1);//取出第一个字符
			shortName += firstLetter;
		}
		if(!"".equals(shortName)){
			//如果该缩写在同义词结果中以单独的词存在 则说明是别名
			for(String synonym:list){
				//检查提取的别名中是否存在这个缩写形式
				String[] words = synonym.split(" ");
				for(String word:words){
					if(shortName.equals(word)){
						return shortName;
					}
				}
			}
		}
		return "";//表示项目名的缩写不是项目的别名
	}
	
	//项目名是缩写 判断有没有相应的全称
	public String getTotalProjectName(String name, List<String> list){
		if(RegexHandler.othersExceptEng(name.toLowerCase())){
			//表示项目名含有除小写英文字幕外的其他字符
			return "";
		}else{
			//将项目名一个字符一个字符分隔开
			String result = StringHandler.getFullName(name, list);
			return result;//如果没有返回""
		}
	}
	
	/**
	 * 对一个新软件的全部操作
	 * @param args
	 */
	@Transactional(propagation=Propagation.REQUIRED)
	public void handleNewProject(GatherProjectsModel model, boolean updateOrNot){
		//提取项目的homepage
		String homepage = model.getHomepage();
		//如果是null 不需要查找相同homepage的项目
		List<GatherProjectsModel> sameHomepageList = new ArrayList<GatherProjectsModel>();
		if(homepage != null && !"".equals(homepage)){
			homepage = homepage.trim();
			//查找所有相同homepage字段的项目
			//判断homepage是否有http://或https://
			int index_http = homepage.indexOf("http://");
			int index_https = homepage.indexOf("https://");
			if(index_http == 0){
				homepage = homepage.substring(index_http + 7);	//去除http://
			}else if(index_https == 0){
				homepage = homepage.substring(index_https + 8);	//去除https://
			}else{
				//表示不是以http或https开头
			}
			
			if((homepage.lastIndexOf("/") == homepage.length()-1) && (homepage.length() != 0)){
				//如果homepage的最后一个字符是/ 需要去除掉
				homepage = homepage.substring(0, homepage.length() - 1);
			}
			if(homepage.length() > 0){
				//为homepage添加http或https头
				String homepage1 = "http://" + homepage;
				String homepage2 = "https://" + homepage;
				String homepage3 = "http://" + homepage + "/";
				String homepage4 = "https://" + homepage + "/";
				
				sameHomepageList = gatherDao.selectGPMBySameHomePage(gatherProjectsTableName, homepage, homepage1, homepage2, homepage3, homepage4, model.getId(), model.getSource());//认为一定是相同的项目
				
				//查看sameHomepageList和当前的model存不存在来自同一社区的项目 如果存在则认为homepage匹配有问题 就将sameHomepageList清空
				
				
				for(GatherProjectsModel m : sameHomepageList){
					logger.info("project " + model.getId() + " and " + m.getId() + " match by homepage");
				}
			}
		}
		
		//如果项目名由多个单词组成，并且相同 则认为两个项目相同!cvs-fast-export 连字符特殊情况
		List<GatherProjectsModel> sameNameList = new ArrayList<GatherProjectsModel>();
		if(StringHandler_ProjectName.getWordNum(model.getName()) > 1){
			//表示项目名由多个单词组成
			sameNameList = gatherDao.selectGPMBySameName(gatherProjectsTableName, model.getName(), model.getId());	//找到表中相同名称的项目
			
			for(GatherProjectsModel m : sameNameList){
				logger.info("project " + model.getId() + " and " + m.getId() + " match by name");
			}
		}
		
		//提取项目的同义词
		List<String> synonymList = getSynonyms(model);//获取项目的同义词
		List<GatherProjectsModel> matchedProjectList = getRelatedGatherProjects(model, synonymList);//在之前已经处理过的项目中找到有相同同义词的项目列表
		
		for(GatherProjectsModel m : matchedProjectList){
			logger.info("project " + model.getId() + " and " + m.getId() + " match by synonyms");
		}
		
		for(int i = 0; i < matchedProjectList.size(); i++){
			GatherProjectsModel model2 = matchedProjectList.get(i);
			if(!areTwoProjectsTheSame(model, model2)){
				matchedProjectList.remove(i);//对于不是同一项目的在匹配列表中去除掉
				i--;
			}
		}
		
		for(GatherProjectsModel m : matchedProjectList){
			logger.info("project " + model.getId() + " and " + m.getId() + " match by synonyms and verify");
		}
		
		//需要将两个相同项目的集合中的项目进行合并
		matchedProjectList = mergeTwoList(sameHomepageList, matchedProjectList);
		matchedProjectList = mergeTwoList(sameNameList, matchedProjectList);
		
		/**
		 * 对项目同义词表进行合并,并添加项目之间的关联(匹配完之后的所有操作)
		 * 是不是相同的项目都做相同的操作 因为在edd_relations表中不论是删除还是更新都是先删除再插入
		 */
		handleAfterMatch(model,synonymList,matchedProjectList, updateOrNot);
	}
	
	/**
	 * 判断edd_relations中的记录合并到一起会不会导致项目中有相同的项目匹配到一起
	 * 如果存在：返回null 如果不存在：返回匹配的结果
	 */
	//!!!!!!!!!同一社区直接删除
	public String hasDiffProjFromSameCommunity(List<EddRelations> list){
		String result = ",";
		Map<Integer, Integer> map_project = new HashMap<Integer, Integer>();
		Map<String, Integer> map_source = new HashMap<String, Integer>();
		for(EddRelations relation:list){
			//统计不同的项目id
			String gather_projects_ids = relation.getGather_projects_ids();
			gather_projects_ids = gather_projects_ids.substring(1, gather_projects_ids.length() - 1);
			String[] idArray = gather_projects_ids.split(",");
			for(String id:idArray)
				map_project.put(Integer.parseInt(id), 1);
		}
		Set<Integer> idSet = map_project.keySet();
		Iterator it = idSet.iterator();
		while(it.hasNext()){
			int id = (int)it.next();
			GatherProjectsModel model = gatherDao.selectGPMById(gatherProjectsTableName, id);
			String source = model.getSource();
			if(map_source.containsKey(source)){
				return null;//表示有重复社区的项目
			}else{
				map_source.put(source, 1);
				result += id + ",";
			}
		}
		return result;
	}
	
	/**
	 * 删除edd_relations表中对应id的记录 如果有osp_id则也要删除对应的open_source_projects表中的记录
	 */
	public void deleteExistEddRelations(EddRelations relation){
		//删除之前对应的relation记录
		int edd_relation_id = relation.getId();
		dbSource.deleteEddRelationsItem(eddRelationTableName, edd_relation_id);
	}
	
	/**
	 * 将两个list<GatherProjectsModel>进行合并
	 */
	public List<GatherProjectsModel> mergeTwoList(List<GatherProjectsModel> list1, List<GatherProjectsModel> list2){
		List<GatherProjectsModel> result = new ArrayList<GatherProjectsModel>();
		Map<GatherProjectsModel, Integer> map = new HashMap<GatherProjectsModel, Integer>();
		for(GatherProjectsModel model:list1){
			if(!map.containsKey(model))
				map.put(model, 1);
		}
		for(GatherProjectsModel model:list2){
			if(!map.containsKey(model))
				map.put(model, 1);
		}
		Set<GatherProjectsModel> keySet = map.keySet();
		Iterator it = keySet.iterator();
		while(it.hasNext()){
			result.add((GatherProjectsModel) it.next());
		}
		return result;
	}
	
	/**
	 * 比较两个list中相同个数
	 */
	public List<String> getSameStringsInList(List<String> list1, List<String> list2){
		List<String> result = new ArrayList<String>();
		for(int i = 0; i < list1.size(); i++){
			String str = list1.get(i);
			String sameString = theMostSimilarString(str, list2);
			if(sameString != null)
				result.add(sameString);
		}
		return result;
	}
	
	
	/**
	 * 判断是不是有来自同一社区的项目
	 */
	
}
